{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 梯度：由全部变量的偏导数汇总而成的向量称为梯度\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "def numerical_gradient(f, x):\n",
    "    h = 1e-4 # 0.0001\n",
    "    grad = np.zeros_like(x)\n",
    "\n",
    "    for idx in range(x.size):\n",
    "        tmp_val = x[idx]\n",
    "        # 计算f(x+h)\n",
    "        x[idx] = tmp_val + h\n",
    "        fxh1 = f(x)\n",
    "        # 计算f(x-h)\n",
    "        x[idx] = tmp_val - h\n",
    "        fxh2 = f(x)\n",
    "\n",
    "        grad[idx] = (fxh1 - fxh2) / (2 * h)\n",
    "        x[idx] = tmp_val\n",
    "\n",
    "    return grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "# 多元函数\n",
    "\n",
    "def function_2(x):\n",
    "    return np.sum(x ** 2)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6. 8.]\n",
      "[0. 4.]\n",
      "[6. 0.]\n"
     ]
    }
   ],
   "source": [
    "# 计算梯度\n",
    "print(numerical_gradient(function_2, np.array([3.0, 4.0])))\n",
    "print(numerical_gradient(function_2, np.array([0.0, 2.0])))\n",
    "print(numerical_gradient(function_2, np.array([3.0, 0.0])))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [],
   "source": [
    "# 梯度下降法\n",
    "\n",
    "def gradient_descent(f, init_x, lr=0.01, step_num=100):\n",
    "    x = init_x\n",
    "\n",
    "    for i in range(step_num):\n",
    "        grad = numerical_gradient(f, x)\n",
    "        x -= lr * grad\n",
    "        print(f(x))\n",
    "\n",
    "    return x"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15.99999999999875\n",
      "10.239999999981373\n",
      "6.5535999999845505\n",
      "4.194303999986998\n",
      "2.6843545599914576\n",
      "1.717986918394193\n",
      "1.0995116277721977\n",
      "0.7036874417742796\n",
      "0.4503599627355467\n",
      "0.2882303761507752\n",
      "0.18446744073649915\n",
      "0.1180591620713699\n",
      "0.07555786372568082\n",
      "0.04835703278443876\n",
      "0.030948500982042153\n",
      "0.019807040628507862\n",
      "0.012676506002244754\n",
      "0.00811296384143647\n",
      "0.005192296858519232\n",
      "0.003323069989452288\n",
      "0.0021267647932494446\n",
      "0.0013611294676796398\n",
      "0.0008711228593149723\n",
      "0.000557518629961583\n",
      "0.0003568119231754152\n",
      "0.00022835963083226647\n",
      "0.00014615016373265095\n",
      "9.353610478889676e-05\n",
      "5.986310706489384e-05\n",
      "3.8312388521531945e-05\n",
      "2.4519928653780434e-05\n",
      "1.5692754338419486e-05\n",
      "1.004336277658848e-05\n",
      "6.427752177016627e-06\n",
      "4.1137613932906445e-06\n",
      "2.632807291706011e-06\n",
      "1.6849966666918467e-06\n",
      "1.0783978666827816e-06\n",
      "6.901746346769799e-07\n",
      "4.4171176619326703e-07\n",
      "2.8269553036369085e-07\n",
      "1.8092513943276212e-07\n",
      "1.1579208923696776e-07\n",
      "7.410693711165936e-08\n",
      "4.742843975146199e-08\n",
      "3.0354201440935676e-08\n",
      "1.9426688922198835e-08\n",
      "1.2433080910207257e-08\n",
      "7.957171782532644e-09\n",
      "5.092589940820891e-09\n",
      "3.2592575621253703e-09\n",
      "2.085924839760237e-09\n",
      "1.3349918974465515e-09\n",
      "8.54394814365793e-10\n",
      "5.468126811941074e-10\n",
      "3.499601159642287e-10\n",
      "2.2397447421710636e-10\n",
      "1.4334366349894805e-10\n",
      "9.173994463932674e-11\n",
      "5.871356456916911e-11\n",
      "3.7576681324268233e-11\n",
      "2.4049076047531666e-11\n",
      "1.5391408670420274e-11\n",
      "9.850501549068967e-12\n",
      "6.304320991404136e-12\n",
      "4.034765434498654e-12\n",
      "2.582249878079141e-12\n",
      "1.652639921970649e-12\n",
      "1.0576895500612165e-12\n",
      "6.769213120391767e-13\n",
      "4.3322963970507253e-13\n",
      "2.7726696941124664e-13\n",
      "1.7745086042319708e-13\n",
      "1.1356855067084696e-13\n",
      "7.268387242934229e-14\n",
      "4.651767835477921e-14\n",
      "2.97713141470584e-14\n",
      "1.9053641054117485e-14\n",
      "1.2194330274634799e-14\n",
      "7.804371375766069e-15\n",
      "4.994797680490399e-15\n",
      "3.1966705155138836e-15\n",
      "2.045869129928833e-15\n",
      "1.3093562431544855e-15\n",
      "8.379879956188697e-16\n",
      "5.36312317196085e-16\n",
      "3.4323988300549014e-16\n",
      "2.196735251235485e-16\n",
      "1.405910560790581e-16\n",
      "8.997827589061819e-17\n",
      "5.758609657000494e-17\n",
      "3.685510180481421e-17\n",
      "2.358726515508575e-17\n",
      "1.5095849699260665e-17\n",
      "9.661343807527544e-18\n",
      "6.183260036816887e-18\n",
      "3.957286423560109e-18\n",
      "2.532663311082945e-18\n",
      "1.6209045190929581e-18\n",
      "1.0373788922158197e-18\n",
      "[-6.11110793e-10  8.14814391e-10]\n"
     ]
    }
   ],
   "source": [
    "# 示例：使用梯度法求解f(x0 + x1) = x0**2 + x1**2的最小值\n",
    "\n",
    "def function_2(x):\n",
    "    return x[0]**2 + x[1]**2\n",
    "\n",
    "init_x = np.array([-3.0, 4.0])\n",
    "print(gradient_descent(function_2, init_x, lr=1e-1, step_num=100))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "# 神经网络的梯度\n",
    "import sys, os\n",
    "sys.path.append(os.pardir)\n",
    "import numpy as np\n",
    "from common import functions, gradient\n",
    "\n",
    "class simpleNet:\n",
    "    def __init__(self):\n",
    "        self.W = np.random.randn(2, 3)\n",
    "\n",
    "    def predict(self, x):\n",
    "        return np.dot(x, self.W)\n",
    "\n",
    "    def loss(self, x, t):\n",
    "        z = self.predict(x)\n",
    "        y = functions.softmax(z)\n",
    "        loss = functions.cross_entropy_error(y, t)\n",
    "\n",
    "        return loss"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.16046145  1.59814243 -0.18937968]\n",
      " [ 0.16302534 -0.46348719 -0.8385892 ]]\n",
      "2.0284996563386697\n"
     ]
    }
   ],
   "source": [
    "# 计算梯度\n",
    "net = simpleNet()\n",
    "print(net.W)\n",
    "x = np.array([0.6, 0.9])\n",
    "p = net.predict(x)\n",
    "t = np.array([0, 0, 1])\n",
    "print(net.loss(x, t))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.19779527  0.32328476 -0.52108003]\n",
      " [ 0.29669291  0.48492714 -0.78162005]]\n"
     ]
    }
   ],
   "source": [
    "# 梯度计算\n",
    "\n",
    "f = lambda w: net.loss(x, t)\n",
    "\n",
    "dW = gradient.numerical_gradient(f, net.W)\n",
    "print(dW)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}